package dhy.com.clawer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jsoup.nodes.Document;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.IOException;
import java.util.LinkedList;
import java.util.List;

//负责爬取图片
public class BaiduPictureProcess implements PageProcessor
{
   ObjectMapper objectMapper=new ObjectMapper();

    private final List<String> imgsUrl=new LinkedList<>();

    //站点配置的设置
    Site site=Site.me()
            .setSleepTime(3*1000)
            .setRetryTimes(3)
            .setRetrySleepTime(3*1000)
            .setCharset("UTF-8")
            .addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
            .addHeader("Accept-Encoding","gzip, deflate, sdch, br")
            .addHeader("Accept-Language","zh-CN,zh;q=0.8")
            .addHeader("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

    //页面处理
    @Override
    public void process(Page page)
    {
     //获取所有的img标签
     Document document = page.getHtml().getDocument();
     String imgJson = document.select("body").text();
     try {
      JsonNode jsonNode = objectMapper.readTree(imgJson);
      JsonNode thumbURL = jsonNode.get("data");
      if(thumbURL.isArray())
      {
       //准备一个数组，存放图片url
       for (JsonNode node : thumbURL) {
        JsonNode hoverURL = node.get("thumbURL");
        if(hoverURL==null)continue;
        String imgUrl = hoverURL.asText();
        imgsUrl.add(imgUrl);
       }
       page.putField("url",imgsUrl);
      }
     } catch (IOException e) {
      e.printStackTrace();
     }
    }

    @Override
    public Site getSite()
    {
        return site;
    }

}
